<!Doctype html>
<html>

<head>
    <title>oixan的博客</title>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta http-equiv="Content-Type" content="text/html" ; charset="utf-8">
    <link rel="stylesheet" type="text/css" href="../css/blog.css">
    <link rel="stylesheet" type="text/css" href="./bcss/swiper.min.css">
</head>

<body>
    <div class="head">
        <div class="headnav">
            <div class="naveframe">
                <ul class="navlist">
                    <li>
                        <a href="../index.html">
                            主页
                        </a>
                    </li>
                    <li>
                        <a href="../bloglist.html">
                            博客
                        </a>
                    </li>
                    <li>
                        <a href="../about.html">
                            关于
                        </a>
                    </li>
                </ul>
            </div>
        </div>

        <div class="headtext">
            <div class="headtext1">世界很简单，人类很复杂。</div>
            <div class="headtext2">浮生若梦，寻寻觅觅。</div>
        </div>
    </div>

    <div class="contents-all">
        <div class="contents">

            <div class="frame">
                <div class="title">
                    <h3>python爬虫库looter用法</h3>
                </div>
                <div class="dtime">时间：2018-07-31</div>
                <div class="detail">

                    <p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;"></span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">import looter as lt</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">from concurrent import futures</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">from pprint import pprint</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">from fake_useragent import UserAgent</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">import time</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">import requests</span>
</p>
<p>
    <br/>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">domain=&#39;https://www.woyaogexing.com&#39;</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">total=[]</span>
</p>
<p>
    <br/>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">def save_imgs(url):</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; headers = {&#39;User-Agent&#39;: UserAgent().random}</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; for img in url:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; if img[0:2]==&#39;//&#39;:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; img=&#39;http:&#39;+img</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; elif url.startswith(&#39;http&#39;):</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; img=img</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; else:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; img=&#39;https://&#39;+img</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; img_name=img.split(&#39;/&#39;)[-1]</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; with open(&#39;d:/pythoncode/pic/%s&#39;%img_name,&#39;wb&#39;) as f:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; f.write(requests.get(img,headers,timeout=60).content)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; print(&#39;Img:%s;Saved\n&#39;%img_name)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp;&nbsp;</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">def crawl_href(url):</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; tree=lt.fetch(url)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; items=tree.cssselect(&#39;a.img&#39;)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; for item in items:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; data=dict()</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; data[&#39;href&#39;]=item.get(&#39;href&#39;)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; #pprint(data)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; total.append(data)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp;&nbsp;</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">def href_get():</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; tasklist=[f&#39;{domain}/tupian/weimei/index_{i}.html&#39; for i in range(4,5)]</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; with futures.ThreadPoolExecutor(20) as executor:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; executor.map(crawl_href,tasklist)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp;&nbsp;</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">def crawl_img(url):</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; tree=lt.fetch(url)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; imgs=tree.cssselect(&#39;img.lazy&#39;)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; for img in imgs:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; imgs=[img.get(&#39;src&#39;)]</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; save_imgs(imgs)</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp;&nbsp;</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">def img_get():</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; tasklist=[f&#39;{domain}{url[&quot;href&quot;]}&#39; for url in total]</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; with futures.ThreadPoolExecutor(20) as executor:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; &nbsp; &nbsp; executor.map(crawl_img,tasklist)</span>
</p>
<p>
    <br/>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">if __name__==&#39;__main__&#39;:</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; href_get()</span>
</p>
<p>
    <span style="font-family: 宋体, SimSun; font-size: 14px;">&nbsp; &nbsp; img_get()</span>
</p>
<p>
    &nbsp; &nbsp; &nbsp;
</p>
                        


                </div>
                <a href="javascript:void(0)" onclick="showPg()">返回</a>
            </div>

        </div>
    </div>
    <div class="footer">
        <div class="footertext">©2018,oixan博客</div>
    </div>
    <span id="backtop" onclick="topFunction()">返回顶部</span>
    <script src="./bjs/aes.js"></script>
</body>
<script  src="../js/fanhuiye.js"></script>

</html>